package org.tangerinejunction.imstats.data;

import java.util.ArrayList;
import java.util.Calendar;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.TreeSet;

import org.tangerinejunction.imstats.logs.LogFile;
import org.tangerinejunction.imstats.stats.AccountStatistics;
import org.tangerinejunction.imstats.stats.EntityStatistics;
import org.tangerinejunction.imstats.stats.FileStatistics;
import org.tangerinejunction.imstats.util.Cycle;
import org.tangerinejunction.imstats.util.TimeRange;

import prefuse.data.Schema;
import prefuse.data.Table;
import prefuse.data.Tuple;
import prefuse.data.event.ExpressionListener;
import prefuse.data.expression.Expression;
import prefuse.data.expression.ExpressionVisitor;
import prefuse.data.util.TableIterator;

/**
 * This utility helps to create columns for data tables.
 * 
 * @author meganthorsen
 */
public class TableFactory 
{
	public static final String FILE_COUNT_COL_NAME = "# files";
	public static final String NORM_FILE_COUNT_COL_NAME = "% files";
	public static final String FILE_SIZE_COL_NAME = "File size";
	public static final String NORM_FILE_SIZE_COL_NAME = "% size";
	public static final String FILE_DENSITY_COL_NAME = "File density";
	public static final String NORM_FILE_DENSITY_COL_NAME = "% density";
	
	public static final String CONSISTENCY_COL_NAME = "Consistency";
	public static final String FREQUENT_LENGTH_COL_NAME = "Solidity";
	public static final String ROW_COL_NAME = "Row";
	
	public static final String CONVERSATIONS_PER_DAY_NAME = "Conv/day";
    public static final String DAYS_BETWEEN_CONVERSATION_NAME = "Avg days between";
	public static final String MAX_DAYS_BETWEEN_NAME = "Max days between";
	public static final String PROB_I_STARTED_NAME = "% I started";
	public static final String SIZE_I_STARTED_NAME = "Size I started";
    public static final String STARTED_TIMES_VOLUME_NAME = "Imbalance";
	public static final String NUM_DAYS_IN_DATASET_NAME = "Num days";
	
	/**
	 * 
	 * @param colName	the name of the column to add
	 * @param dataTable	the Table to which we want to add a column
	 * @param dateRange	the DateRange over which the column is valid
	 * @param isIncrement	whether the column is incremented
	 */
	public static void addColumnToTable(String colName, Table dataTable, TimeRange dateRange, boolean isIncrement)
	{
		if (colName.equals(FILE_COUNT_COL_NAME))
			addFileCountColumnToTable(dataTable, dateRange, isIncrement);
		else if (colName.equals(NORM_FILE_COUNT_COL_NAME))
			addNormalizedFileCountColumnToTable(dataTable, dateRange, isIncrement);
		else if (colName.equals(FILE_SIZE_COL_NAME))
			addFileSizeColumnToTable(dataTable, dateRange, isIncrement);
		else if (colName.equals(NORM_FILE_SIZE_COL_NAME))
			addNormalizedFileSizeColumnToTable(dataTable, dateRange, isIncrement);
		else if (colName.equals(FILE_DENSITY_COL_NAME))
			addFileDensityColumnToTable(dataTable, dateRange, isIncrement);
		else if (colName.equals(NORM_FILE_DENSITY_COL_NAME))
			addNormalizedFileDensityColumnToTable(dataTable, dateRange, isIncrement);
		else if (colName.equals(CONVERSATIONS_PER_DAY_NAME))
		    addConversationsPerDayColumnToTable(dataTable, dateRange, isIncrement);
		else if (colName.equals(MAX_DAYS_BETWEEN_NAME))
		    addMaxDaysBetweenColumnToTable(dataTable, dateRange, isIncrement);
        else if (colName.equals(PROB_I_STARTED_NAME))
            addProbIStartedColumnToTable(dataTable, dateRange, isIncrement);
        else if (colName.equals(SIZE_I_STARTED_NAME))
            addSizeIStartedColumnToTable(dataTable, dateRange, isIncrement);
        else if (colName.equals(NUM_DAYS_IN_DATASET_NAME))
            addNumDaysInDatasetColumnToTable(dataTable, dateRange, isIncrement);
	}
	
	/**
	 * 
	 * @param dataTable	the Table from which we want the rows
	 * @return	the array of indices associated with the Table rows
	 */
	private static int[] getRows(Table dataTable)
	{
		TableIterator rowIter = dataTable.iterator();
		int[] rows = new int[dataTable.getRowCount()];
		int index = 0;
		while (rowIter.hasNext())
		{
			rows[index] = rowIter.nextInt();
			index++;
		}
		return rows;
	}
	
	/**
	 * 
	 * @param dataTable	the Table to which we want to add a file count column
	 * @param dateRange	the DateRange over which the column will be valid
	 * @param isIncrement	whether the column is incremented
	 */
	public static void addFileCountColumnToTable(Table dataTable, TimeRange dateRange, boolean isIncrement)
	{
		String colTitle = isIncrement ? dateRange.toLabelString() : FILE_COUNT_COL_NAME;
		if (dataTable.getColumn(colTitle) != null)
			dataTable.removeColumn(colTitle);
		dataTable.addColumn(colTitle, Integer.class);
		int[] rows = getRows(dataTable);
		for (int i = 0; i < rows.length; i++)
		{
			int row = rows[i];
			if (dataTable.getTuple(row) instanceof Account)
			{
				Account account = (Account)dataTable.getTuple(row);
				ArrayList<LogFile> logFiles = (ArrayList<LogFile>) account.get(Account.LOG_FILES_KEY);
				if (logFiles == null)
					System.err.println("NULL log files for account " + account.getName());
				int fileCount = 0;
				for (LogFile logFile: logFiles)
				{
					if (dateRange.contains(logFile.getDate()))
						fileCount++;
				}
				dataTable.set(row, colTitle, fileCount);
			}
			else // it's an Entity
			{
				Entity entity = (Entity)dataTable.getTuple(row);
				AccountSet accounts = (AccountSet)entity.get(Entity.ACCOUNTS_KEY);
				ArrayList<LogFile> logFiles = accounts.getLogFiles();
				int fileCount = 0;
				for (LogFile logFile: logFiles)
				{
					if (dateRange.contains(logFile.getDate()))
						fileCount++;
				}
				dataTable.set(row, colTitle, fileCount);
			}
		}
	}
	
	/**
	 * 
	 * @param dataTable	the Table to which we want to add a normalized file count column
	 * @param dateRange	the DateRange over which the column will be valid
	 * @param isIncrement	whether the column is incremented
	 */
	public static void addNormalizedFileCountColumnToTable(Table dataTable, TimeRange dateRange, boolean isIncrement)
	{
		String colTitle = isIncrement ? dateRange.toLabelString() : NORM_FILE_COUNT_COL_NAME;
		if (dataTable.getColumn(colTitle) != null)
			dataTable.removeColumn(colTitle);
		dataTable.addColumn(colTitle, Double.class);
		//double totalCount = DataLib.sum(dataTable.tuples(), FILE_COUNT_COL_NAME);
		double totalCount = 0;
		int[] rows = getRows(dataTable);
		for (int i = 0; i < rows.length; i++)
		{
			int row = rows[i];
			if (dataTable.getTuple(row) instanceof Account)
			{
				Account account = (Account)dataTable.getTuple(row);
				ArrayList<LogFile> logFiles = (ArrayList<LogFile>) account.get(Account.LOG_FILES_KEY);
				int fileCount = 0;
				for (LogFile logFile: logFiles)
				{
					if (dateRange.contains(logFile.getDate()))
						fileCount++;
				}
				//dataTable.set(row, NORM_FILE_COUNT_COL_NAME, ((double)fileCount)/totalCount);
				dataTable.set(row, colTitle, (double)fileCount);
				totalCount += fileCount;
			}
			else // it's an Entity
			{
				Entity entity = (Entity)dataTable.getTuple(row);
				AccountSet accounts = (AccountSet)entity.get(Entity.ACCOUNTS_KEY);
				ArrayList<LogFile> logFiles = accounts.getLogFiles();
				int fileCount = 0;
				for (LogFile logFile: logFiles)
				{
					if (dateRange.contains(logFile.getDate()))
						fileCount++;
				}
				//dataTable.set(row, NORM_FILE_COUNT_COL_NAME, ((double)fileCount)/totalCount);
				dataTable.set(row, colTitle, (double)fileCount);
				totalCount += fileCount;
			}
		}
		for (int i = 0; i < rows.length; i++)
		{
			int row = rows[i];
			Double fileCount = (Double)dataTable.get(row, colTitle);
			dataTable.set(row, colTitle, totalCount == 0 ? 0 : 100*fileCount/totalCount);
		}
	}
	
	/**
	 * 
	 * @param dataTable	the Table to which we want to add a file size column
	 * @param dateRange	the DateRange over which the column will be valid
	 * @param isIncrement	whether the column is incremented
	 */
	public static void addFileSizeColumnToTable(Table dataTable, TimeRange dateRange, boolean isIncrement)
	{
		String colTitle = isIncrement ? dateRange.toLabelString() : FILE_SIZE_COL_NAME;
		if (dataTable.getColumn(colTitle) != null)
			dataTable.removeColumn(colTitle);
		dataTable.addColumn(colTitle, Integer.class);
		int[] rows = getRows(dataTable);
		for (int i = 0; i < rows.length; i++)
		{
			int row = rows[i];
			if (dataTable.getTuple(row) instanceof Account)
			{
				Account account = (Account)dataTable.getTuple(row);
				ArrayList<LogFile> logFiles = (ArrayList<LogFile>) account.get(Account.LOG_FILES_KEY);
				int fileSizes = 0;
				for (LogFile logFile: logFiles)
				{
					if (dateRange.contains(logFile.getDate()))
						fileSizes += FileStatistics.getFileSize(logFile);
				}
				dataTable.set(row, colTitle, fileSizes);
			}
			else // it's an Entity
			{
				Entity entity = (Entity)dataTable.getTuple(row);
				AccountSet accounts = (AccountSet)entity.get(Entity.ACCOUNTS_KEY);
				ArrayList<LogFile> logFiles = accounts.getLogFiles();
				int fileSizes = 0;
				for (LogFile logFile: logFiles)
				{
					if (dateRange.contains(logFile.getDate()))
						fileSizes += FileStatistics.getFileSize(logFile);
				}
				dataTable.set(row, colTitle, fileSizes);
			}
		}
	}
	
	/**
	 * 
	 * @param dataTable	the Table to which we want to add a normalized file size column
	 * @param dateRange	the DateRange over which the column will be valid
	 * @param isIncrement	whether the column is incremented
	 */
	public static void addNormalizedFileSizeColumnToTable(Table dataTable, TimeRange dateRange, boolean isIncrement)
	{
		String colTitle = isIncrement ? dateRange.toLabelString() : NORM_FILE_SIZE_COL_NAME;
		if (dataTable.getColumn(colTitle) != null)
			dataTable.removeColumn(colTitle);
		dataTable.addColumn(colTitle, Double.class);
		//double totalSize = DataLib.sum(dataTable.tuples(), FILE_SIZE_COL_NAME);
		double totalSize = 0;
		int[] rows = getRows(dataTable);
		for (int i = 0; i < rows.length; i++)
		{
			int row = rows[i];
			if (dataTable.getTuple(row) instanceof Account)
			{
				Account account = (Account)dataTable.getTuple(row);
				ArrayList<LogFile> logFiles = (ArrayList<LogFile>) account.get(Account.LOG_FILES_KEY);
				int fileSizes = 0;
				for (LogFile logFile: logFiles)
				{
					if (dateRange.contains(logFile.getDate()))
						fileSizes += FileStatistics.getFileSize(logFile);
				}
				//dataTable.set(row, NORM_FILE_SIZE_COL_NAME, ((double)fileSizes)/totalSize);
				dataTable.set(row, colTitle, (double)fileSizes);
				totalSize += fileSizes;
			}
			else // it's an Entity
			{
				Entity entity = (Entity)dataTable.getTuple(row);
				AccountSet accounts = (AccountSet)entity.get(Entity.ACCOUNTS_KEY);
				ArrayList<LogFile> logFiles = accounts.getLogFiles();
				int fileSizes = 0;
				for (LogFile logFile: logFiles)
				{
					if (dateRange.contains(logFile.getDate()))
						fileSizes += FileStatistics.getFileSize(logFile);
				}
				//dataTable.set(row, NORM_FILE_SIZE_COL_NAME, ((double)fileSizes)/totalSize);
				dataTable.set(row, colTitle, (double)fileSizes);
				totalSize += fileSizes;
			}
		}
		for (int i = 0; i < rows.length; i++)
		{
			int row = rows[i];
			Double fileSize = (Double)dataTable.get(row, colTitle);
			dataTable.set(row, colTitle, totalSize == 0 ? 0 : 100*fileSize/totalSize);
		}
	}
	
	/**
	 * 
	 * @param dataTable	the Table to which we want to add a file density column
	 * @param dateRange	the DateRange over which the column will be valid
	 * @param isIncrement	whether the column is incremented
	 */
	public static void addFileDensityColumnToTable(Table dataTable, TimeRange dateRange, boolean isIncrement)
	{
		String colTitle = isIncrement ? dateRange.toLabelString() : FILE_DENSITY_COL_NAME;
		if (dataTable.getColumn(colTitle) != null)
			dataTable.removeColumn(colTitle);
		dataTable.addColumn(colTitle, Double.class);
		int[] rows = getRows(dataTable);
		for (int i = 0; i < rows.length; i++)
		{
			int row = rows[i];
			if (dataTable.getTuple(row) instanceof Account)
			{
				Account account = (Account)dataTable.getTuple(row);
				ArrayList<LogFile> logFiles = (ArrayList<LogFile>) account.get(Account.LOG_FILES_KEY);
				int fileSizes = 0;
				for (LogFile logFile: logFiles)
				{
					if (dateRange.contains(logFile.getDate()))
						fileSizes += FileStatistics.getFileSize(logFile);
				}
				double fileCount = AccountStatistics.getFileCount(account, dateRange);
				dataTable.set(row, colTitle, fileCount > 0 ?
						new Double((double)fileSizes/fileCount) : 0);
			}
			else // it's an Entity
			{
				Entity entity = (Entity)dataTable.getTuple(row);
				AccountSet accounts = (AccountSet)entity.get(Entity.ACCOUNTS_KEY);
				ArrayList<LogFile> logFiles = accounts.getLogFiles();
				int fileSizes = 0;
				for (LogFile logFile: logFiles)
				{
					if (dateRange.contains(logFile.getDate()))
						fileSizes += FileStatistics.getFileSize(logFile);
				}
				double fileCount = EntityStatistics.getFileCount(entity, dateRange);
				dataTable.set(row, colTitle, (fileCount > 0 ?
						new Double((double)fileSizes/fileCount) : 0));
			}
		}
	}
	
	/**
	 * 
	 * @param dataTable	the Table to which we want to add a normalized file density column
	 * @param dateRange	the DateRange over which the column will be valid
	 * @param isIncrement	whether the column is incremented
	 */
	public static void addNormalizedFileDensityColumnToTable(Table dataTable, TimeRange dateRange, boolean isIncrement)
	{
		String colTitle = isIncrement ? dateRange.toLabelString() : NORM_FILE_DENSITY_COL_NAME;
		if (dataTable.getColumn(colTitle) != null)
			dataTable.removeColumn(colTitle);
		dataTable.addColumn(colTitle, Double.class);
		double totalDensity = 0;
		int[] rows = getRows(dataTable);
		for (int i = 0; i < rows.length; i++)
		{
			int row = rows[i];
			if (dataTable.getTuple(row) instanceof Account)
			{
				Account account = (Account)dataTable.getTuple(row);
				ArrayList<LogFile> logFiles = (ArrayList<LogFile>) account.get(Account.LOG_FILES_KEY);
				int fileSizes = 0;
				for (LogFile logFile: logFiles)
				{
					if (dateRange.contains(logFile.getDate()))
						fileSizes += FileStatistics.getFileSize(logFile);
				}
				double fileCount = AccountStatistics.getFileCount(account, dateRange);
				double value = fileCount > 0 ?
						new Double((double)fileSizes/fileCount) : 0;
				dataTable.set(row, colTitle, value);
				totalDensity += value;
			}
			else // it's an Entity
			{
				Entity entity = (Entity)dataTable.getTuple(row);
				AccountSet accounts = (AccountSet)entity.get(Entity.ACCOUNTS_KEY);
				ArrayList<LogFile> logFiles = accounts.getLogFiles();
				int fileSizes = 0;
				for (LogFile logFile: logFiles)
				{
					if (dateRange.contains(logFile.getDate()))
						fileSizes += FileStatistics.getFileSize(logFile);
				}
				double fileCount = EntityStatistics.getFileCount(entity, dateRange);
				double value = fileCount > 0 ?
						new Double((double)fileSizes/fileCount) : 0;
				dataTable.set(row, colTitle, value);
				totalDensity += value;
			}
		}
		for (int i = 0; i < rows.length; i++)
		{
			int row = rows[i];
			Double fileDensity = (Double)dataTable.get(row, colTitle);
			dataTable.set(row, colTitle, totalDensity == 0 ? 0 : 100*fileDensity/totalDensity);
		}
	}
	
	/**
	 * 
	 * @param dataTable	the Table in which we want to calculate the consistency column
	 * @param valueKey	the name of the variable over which we want to calculate the consistency, also the name of the associated table column
	 * @param startIndex	the first column index over which we want to calculate consistency
	 * @param endIndex		the final column index over which we want to calculate consistency
	 */
	public static void calculateConsistencyColumn(Table dataTable, String valueKey, int startIndex, int endIndex)
	{
		int keyColIndex = dataTable.getColumnNumber(valueKey);
		String colTitle = CONSISTENCY_COL_NAME;
		if (dataTable.getColumn(colTitle) == null)
		{
			System.err.println(colTitle + " column does not exist!");
			return;
		}
		int[] rows = getRows(dataTable);
		for (int i = 0; i < rows.length; i++)
		{
			int row = rows[i];
			Double total = 0d;
			Double rowTotal = 0d;
			int n = 0;
			double mean = 0;
			double m2 = 0;
			for (int col = startIndex; col < endIndex; col++)
			{
				Number value = (Number)dataTable.get(row, col);
				rowTotal = rowTotal + value.doubleValue();
			}
			for (int col = startIndex; col < endIndex; col++)
			{
				n++;
				Number value = (Number)dataTable.get(row, col);
				total = total + value.doubleValue();
				double delta = value.doubleValue()/rowTotal - mean;
				mean = mean + delta/n;
				m2 = m2 + delta*(value.doubleValue()/rowTotal - mean);
			}
			Double normVar = new Double(m2/(n-1));
			if (normVar.isNaN())
				normVar = 0.0;
			else
				normVar = Math.round(100000.0 * (1.0 - normVar/.125))/1000.0;
			dataTable.set(row, colTitle, normVar);
		}
	}
	
	/**
	 * 
	 * @param dataTable	the Table to which we want to add a frequent length column
	 * @param dateRange	the DateRange over which the column will be valid
	 */
	public static void addFrequentLengthColumnToTable(Table dataTable, TimeRange dateRange)
	{
		if (dataTable.getColumn(FREQUENT_LENGTH_COL_NAME) != null)
			dataTable.removeColumn(FREQUENT_LENGTH_COL_NAME);
		//dataTable.addColumn(FREQUENT_LENGTH_COL_NAME, Double.class);
		Expression expr = new Expression()
		{
			public void addExpressionListener(ExpressionListener lstnr) 
			{
				// TODO Auto-generated method stub
				
			}

			public Object get(Tuple t) 
			{
				return new Double(getDouble(t));
			}

			public boolean getBoolean(Tuple t) 
			{
				return false;
			}

			public double getDouble(Tuple t) 
			{
				double normFileCount = (Double)t.get(NORM_FILE_COUNT_COL_NAME)/100;
				double normFileSize = (Double)t.get(NORM_FILE_DENSITY_COL_NAME)/100;
				double rms = Math.sqrt(Math.pow(normFileCount, 2)+Math.pow(normFileSize, 2));
				return rms;
			}

			public float getFloat(Tuple t) 
			{
				return 0;
			}

			public int getInt(Tuple t) 
			{
				return 0;
			}

			public long getLong(Tuple t) 
			{
				return 0;
			}

			public Class getType(Schema s) 
			{
				return Double.class;
			}

			public void removeExpressionListener(ExpressionListener lstnr) 
			{
				// TODO Auto-generated method stub
				
			}

			public void visit(ExpressionVisitor v) 
			{
				// TODO Auto-generated method stub
				
			}
			
		};
		dataTable.addColumn(FREQUENT_LENGTH_COL_NAME, expr);
	}
	

    
    /**
     * 
     * @param dataTable the Table to which we want to add a file count column
     * @param dateRange the DateRange over which the column will be valid
     * @param isIncrement   whether the column is incremented
     */
    public static void addConversationsPerDayColumnToTable(Table dataTable, TimeRange dateRange, boolean isIncrement)
    {
        String colTitle = CONVERSATIONS_PER_DAY_NAME;
        if (dataTable.getColumn(colTitle) != null)
            dataTable.removeColumn(colTitle);
        dataTable.addColumn(colTitle, Double.class);
        String colTitle2 = DAYS_BETWEEN_CONVERSATION_NAME;
        if (dataTable.getColumn(colTitle2) != null)
            dataTable.removeColumn(colTitle2);
        dataTable.addColumn(colTitle2, Double.class);
        int[] rows = getRows(dataTable);
        for (int i = 0; i < rows.length; i++)
        {
            int row = rows[i];
            if (dataTable.getTuple(row) instanceof Account)
            {
                Account account = (Account)dataTable.getTuple(row);
                ArrayList<LogFile> logFiles = (ArrayList<LogFile>) account.get(Account.LOG_FILES_KEY);
                if (logFiles == null)
                    System.err.println("NULL log files for account " + account.getName());
                int fileCount = 0;
                for (LogFile logFile: logFiles)
                {
                    if (dateRange.contains(logFile.getDate()))
                        fileCount++;
                }
                Calendar firstDate = account.getMinLogDate().before(dateRange.getBeginning()) ? dateRange.getBeginning() : account.getMinLogDate();
                TimeRange range = dateRange.getBeginning().before(firstDate)
                                  ? new TimeRange(firstDate, dateRange.getEnd()) 
                                  : new TimeRange(dateRange.getBeginning(), dateRange.getEnd());
                dataTable.set(row, colTitle, fileCount/range.getNumDays());
                if (fileCount == 0)
                    dataTable.set(row, colTitle2, range.getNumDays());
                else
                    dataTable.set(row, colTitle2, range.getNumDays()/fileCount);
            }
            else // it's an Entity
            {
                Entity entity = (Entity)dataTable.getTuple(row);
                AccountSet accounts = (AccountSet)entity.get(Entity.ACCOUNTS_KEY);
                ArrayList<LogFile> logFiles = accounts.getLogFiles();
                int fileCount = 0;
                for (LogFile logFile: logFiles)
                {
                    if (dateRange.contains(logFile.getDate()))
                        fileCount++;
                }
                Calendar firstDate = entity.getMinLogDate().before(dateRange.getBeginning()) ? dateRange.getBeginning() : entity.getMinLogDate();
                TimeRange range = dateRange.getBeginning().before(firstDate)
                                  ? new TimeRange(firstDate, dateRange.getEnd()) 
                                  : new TimeRange(dateRange.getBeginning(), dateRange.getEnd());
                dataTable.set(row, colTitle, fileCount/range.getNumDays());
                if (fileCount == 0)
                    dataTable.set(row, colTitle2, range.getNumDays());
                else
                    dataTable.set(row, colTitle2, range.getNumDays()/fileCount);
            }
        }
    }
    
    /**
     * 
     * @param dataTable the Table to which we want to add a file count column
     * @param dateRange the DateRange over which the column will be valid
     * @param isIncrement   whether the column is incremented
     */
    public static void addMaxDaysBetweenColumnToTable(Table dataTable, TimeRange dateRange, boolean isIncrement)
    {
        String colTitle = MAX_DAYS_BETWEEN_NAME;
        if (dataTable.getColumn(colTitle) != null)
            dataTable.removeColumn(colTitle);
        dataTable.addColumn(colTitle, Double.class);
        int[] rows = getRows(dataTable);
        for (int i = 0; i < rows.length; i++)
        {
            int row = rows[i];
            if (dataTable.getTuple(row) instanceof Account)
            {
                Account account = (Account)dataTable.getTuple(row);
                ArrayList<LogFile> logFiles = (ArrayList<LogFile>) account.get(Account.LOG_FILES_KEY);
                if (logFiles == null)
                    System.err.println("NULL log files for account " + account.getName());
                TreeSet<Calendar> dateSet = new TreeSet();
                HashMap<Calendar, Integer> counts = new HashMap<Calendar, Integer>();
                Calendar firstDate = account.getMinLogDate().before(dateRange.getBeginning()) ? dateRange.getBeginning() : account.getMinLogDate();
                dateSet.add(firstDate);
                counts.put(firstDate, 1);
                if (firstDate != dateRange.getEnd())
                {
                    dateSet.add(dateRange.getEnd());
                    counts.put(dateRange.getEnd(), 1);
                }
                else
                    counts.put(dateRange.getEnd(), 2);
                for (LogFile logFile : logFiles)
                {
                    if (dateRange.contains(logFile.getDate()))
                    {
                        if (!counts.containsKey(logFile.getDate()))
                        {
                            dateSet.add(logFile.getDate());
                            counts.put(logFile.getDate(), 1);
                        }
                        else
                        {
                            counts.put(logFile.getDate(), counts.get(logFile.getDate()) + 1);
                        }
                    }
                }
                Iterator<Calendar> it = dateSet.iterator();
                Calendar previous = null;
                Calendar current = null;
                int fileCount = 0;
                double maxDaysBetween = 0;
                while (it.hasNext())
                {
                    previous = current;
                    current = it.next();
                    fileCount += counts.get(current);
                    if (previous != null)
                    {
                        maxDaysBetween = Math.max(new TimeRange(previous, current).getNumDays(), maxDaysBetween);
                    }
                }
                /*
                int fileCount = 0;
                double maxDaysBetween = 0;
                for (int l = 0; l < logFiles.size(); l++)
                {
                    LogFile prevLogFile = null;
                    LogFile logFile = logFiles.get(l);
                    if (dateRange.contains(logFile.getDate()))
                    {
                        fileCount++;
                        if (fileCount == 1)
                            maxDaysBetween += Math.max(new TimeRange(dateRange.getBeginning(), logFile.getDate()).getNumDays(), maxDaysBetween);
                        else if (fileCount > 1) 
                        {
                            prevLogFile = logFiles.get(l-1);
                            maxDaysBetween = Math.max(new TimeRange(prevLogFile.getDate(), logFile.getDate()).getNumDays(), maxDaysBetween);
                        }
                    }
                    else if (prevLogFile != null)
                    {
                        maxDaysBetween += Math.max(new TimeRange(prevLogFile.getDate(), dateRange.getEnd()).getNumDays(), maxDaysBetween);
                    }
                }
                */
                if (fileCount == 0)
                    dataTable.set(row, colTitle, dateRange.getNumDays());
                else
                    dataTable.set(row, colTitle, maxDaysBetween);
            }
            else // it's an Entity
            {
                Entity entity = (Entity)dataTable.getTuple(row);
                AccountSet accounts = (AccountSet)entity.get(Entity.ACCOUNTS_KEY);
                ArrayList<LogFile> logFiles = accounts.getLogFiles();
                TreeSet<Calendar> dateSet = new TreeSet();
                HashMap<Calendar, Integer> counts = new HashMap<Calendar, Integer>();
                Calendar firstDate = (entity.getMinLogDate() == null || entity.getMinLogDate().before(dateRange.getBeginning())) 
                                     ? dateRange.getBeginning() 
                                     : entity.getMinLogDate();
                dateSet.add(firstDate);
                counts.put(firstDate, 1);
                if (firstDate != dateRange.getEnd())
                {
                    dateSet.add(dateRange.getEnd());
                    counts.put(dateRange.getEnd(), 1);
                }
                else
                    counts.put(dateRange.getEnd(), 2);
                for (LogFile logFile : logFiles)
                {
                    if (dateRange.contains(logFile.getDate()))
                    {
                        if (!counts.containsKey(logFile.getDate()))
                        {
                            dateSet.add(logFile.getDate());
                            counts.put(logFile.getDate(), 1);
                        }
                        else
                        {
                            counts.put(logFile.getDate(), counts.get(logFile.getDate()) + 1);
                        }
                    }
                }
                Iterator<Calendar> it = dateSet.iterator();
                Calendar previous = null;
                Calendar current = null;
                int fileCount = 0;
                double maxDaysBetween = 0;
                while (it.hasNext())
                {
                    previous = current;
                    current = it.next();
                    fileCount += counts.get(current);
                    if (previous != null)
                    {
                        maxDaysBetween = Math.max(new TimeRange(previous, current).getNumDays(), maxDaysBetween);
                    }
                }
                if (fileCount == 0)
                    dataTable.set(row, colTitle, dateRange.getNumDays());
                else
                    dataTable.set(row, colTitle, maxDaysBetween);
            }
        }
    }
    
    /**
     * 
     * @param dataTable the Table to which we want to add a file count column
     * @param dateRange the DateRange over which the column will be valid
     * @param isIncrement   whether the column is incremented
     */
    public static void addProbIStartedColumnToTable(Table dataTable, TimeRange dateRange, boolean isIncrement)
    {
        String colTitle = PROB_I_STARTED_NAME;
        if (dataTable.getColumn(colTitle) != null)
            dataTable.removeColumn(colTitle);
        dataTable.addColumn(colTitle, Double.class);
        String colTitle2 = STARTED_TIMES_VOLUME_NAME;
        if (dataTable.getColumn(colTitle2) != null)
            dataTable.removeColumn(colTitle2);
        dataTable.addColumn(colTitle2, Double.class);
        int[] rows = getRows(dataTable);
        for (int i = 0; i < rows.length; i++)
        {
            int row = rows[i];
            if (dataTable.getTuple(row) instanceof Account)
            {
                Account account = (Account)dataTable.getTuple(row);
                ArrayList<LogFile> logFiles = (ArrayList<LogFile>) account.get(Account.LOG_FILES_KEY);
                if (logFiles == null)
                    System.err.println("NULL log files for account " + account.getName());
                int fileCount = 0;
                int numIStarted = 0;
                int fileSizes = 0;
                for (LogFile logFile: logFiles)
                {
                    if (dateRange.contains(logFile.getDate()))
                    {
                        fileCount++;
                        fileSizes += FileStatistics.getFileSize(logFile);
                        if (logFile.isStartedByMe())
                            numIStarted++;
                    }
                }
                if (fileCount > 0)
                {
                    dataTable.set(row, colTitle, 100.0*numIStarted/fileCount);
                    dataTable.set(row, colTitle2, (0.5-1.0*numIStarted/fileCount)*fileCount);
                }
                else
                {
                    dataTable.set(row, colTitle, 50.0);
                    dataTable.set(row, colTitle2, 0.0);
                }
            }
            else // it's an Entity
            {
                Entity entity = (Entity)dataTable.getTuple(row);
                AccountSet accounts = (AccountSet)entity.get(Entity.ACCOUNTS_KEY);
                ArrayList<LogFile> logFiles = accounts.getLogFiles();
                int fileCount = 0;
                int numIStarted = 0;
                int fileSizes = 0;
                for (LogFile logFile: logFiles)
                {
                    if (dateRange.contains(logFile.getDate()))
                    {
                        fileCount++;
                        fileSizes += FileStatistics.getFileSize(logFile);
                        if (logFile.isStartedByMe())
                            numIStarted++;
                    }
                }
                if (fileCount > 0)
                {
                    dataTable.set(row, colTitle, 100.0*numIStarted/fileCount);
                    dataTable.set(row, colTitle2, (0.5-1.0*numIStarted/fileCount)*fileCount);
                }
                else
                {
                    dataTable.set(row, colTitle, 50.0);
                    dataTable.set(row, colTitle2, 0.0);
                }
            }
        }
    }
    
    /**
     * 
     * @param dataTable the Table to which we want to add a file count column
     * @param dateRange the DateRange over which the column will be valid
     * @param isIncrement   whether the column is incremented
     */
    public static void addSizeIStartedColumnToTable(Table dataTable, TimeRange dateRange, boolean isIncrement)
    {
        String colTitle = SIZE_I_STARTED_NAME;
        if (dataTable.getColumn(colTitle) != null)
            dataTable.removeColumn(colTitle);
        dataTable.addColumn(colTitle, Double.class);
        int[] rows = getRows(dataTable);
        for (int i = 0; i < rows.length; i++)
        {
            int row = rows[i];
            if (dataTable.getTuple(row) instanceof Account)
            {
                Account account = (Account)dataTable.getTuple(row);
                ArrayList<LogFile> logFiles = (ArrayList<LogFile>) account.get(Account.LOG_FILES_KEY);
                if (logFiles == null)
                    System.err.println("NULL log files for account " + account.getName());
                int fileCount = 0;
                int sizeIStarted = 0;
                int sizeTheyStarted = 0;
                for (LogFile logFile: logFiles)
                {
                    if (dateRange.contains(logFile.getDate()))
                    {
                        fileCount++;
                        if (logFile.isStartedByMe())
                            sizeIStarted += FileStatistics.getFileSize(logFile);
                        else
                            sizeTheyStarted += FileStatistics.getFileSize(logFile);
                    }
                }
                if (sizeTheyStarted > 0)
                    dataTable.set(row, colTitle, 1.0*sizeIStarted/sizeTheyStarted);
                else if (fileCount > 0)
                    dataTable.set(row, colTitle, Double.POSITIVE_INFINITY);
                else
                    dataTable.set(row, colTitle, 1.0);
            }
            else // it's an Entity
            {
                Entity entity = (Entity)dataTable.getTuple(row);
                AccountSet accounts = (AccountSet)entity.get(Entity.ACCOUNTS_KEY);
                ArrayList<LogFile> logFiles = accounts.getLogFiles();
                int fileCount = 0;
                int sizeIStarted = 0;
                int sizeTheyStarted = 0;
                for (LogFile logFile: logFiles)
                {
                    if (dateRange.contains(logFile.getDate()))
                    {
                        fileCount++;
                        if (logFile.isStartedByMe())
                            sizeIStarted += FileStatistics.getFileSize(logFile);
                        else
                            sizeTheyStarted += FileStatistics.getFileSize(logFile);
                    }
                }
                if (sizeTheyStarted > 0)
                    dataTable.set(row, colTitle, 1.0*sizeIStarted/sizeTheyStarted);
                else if (fileCount > 0)
                    dataTable.set(row, colTitle, Double.POSITIVE_INFINITY);
                else
                    dataTable.set(row, colTitle, 1.0);
            }
        }
    }
    
    /**
     * 
     * @param dataTable the Table to which we want to add a file count column
     * @param dateRange the DateRange over which the column will be valid
     * @param isIncrement   whether the column is incremented
     */
    public static void addNumDaysInDatasetColumnToTable(Table dataTable, TimeRange dateRange, boolean isIncrement)
    {
        String colTitle = NUM_DAYS_IN_DATASET_NAME;
        if (dataTable.getColumn(colTitle) != null)
            dataTable.removeColumn(colTitle);
        dataTable.addColumn(colTitle, Double.class);
        int[] rows = getRows(dataTable);
        for (int i = 0; i < rows.length; i++)
        {
            int row = rows[i];
            if (dataTable.getTuple(row) instanceof Account)
            {
                Account account = (Account)dataTable.getTuple(row);
                ArrayList<LogFile> logFiles = (ArrayList<LogFile>) account.get(Account.LOG_FILES_KEY);
                if (logFiles == null)
                    System.err.println("NULL log files for account " + account.getName());
                Calendar firstDate = account.getMinLogDate().before(dateRange.getBeginning()) ? dateRange.getBeginning() : account.getMinLogDate();
                TimeRange range = new TimeRange(firstDate, dateRange.getEnd());
                dataTable.set(row, colTitle, range.getNumDays());
            }
            else // it's an Entity
            {
                Entity entity = (Entity)dataTable.getTuple(row);
                AccountSet accounts = (AccountSet)entity.get(Entity.ACCOUNTS_KEY);
                ArrayList<LogFile> logFiles = accounts.getLogFiles();
                TreeSet<Calendar> dateSet = new TreeSet();
                HashMap<Calendar, Integer> counts = new HashMap<Calendar, Integer>();
                Calendar firstDate = entity.getMinLogDate().before(dateRange.getBeginning()) ? dateRange.getBeginning() : entity.getMinLogDate();
                TimeRange range = new TimeRange(firstDate, dateRange.getEnd());
                dataTable.set(row, colTitle, range.getNumDays());
            }
        }
    }
}
